import requests
import lxml.html


def exclude_url_with_prefix(url):
    exclude_prefix = ['board', '2014']
    exclude_prefix += ['/' + s for s in exclude_prefix]
    for prefix in exclude_prefix:
        if url.startswith(prefix):
            return True
    return False

def check_szu(url):
    if not url.startswith('http'):
        return False
    return not 'szu' in url

def exclude_specific_url(url):
    excluded_url = ['/']
    for u in excluded_url:
        if url == u:
            return True
    return False


exclude = [exclude_url_with_prefix, check_szu, exclude_specific_url]

    
def get_inner_link(url):
    response = requests.get(url)
    response.raise_for_status()
    response.encoding = 'gbk'

    etree = lxml.html.fromstring(response.text)
    links = etree.xpath('//a[@href]')

    f = lambda url: (url or [''])[0]
    links = [(f(a.xpath('@href')), f(a.xpath('.//text()'))) for a in links]
    links = [(url, text) for url, text in links \
             if text and not any(pred(url) for pred in exclude)]
    return links 

